<?php
$logPath = '/dahe/log/nginx/oam4api';
//$logPath = '/Users/huyancong/Documents/www/platform/app/Console';
$logPrefix = 'oam4api-';
$logDateFormat = 'Y-m-d';
error_reporting(E_ALL^E_NOTICE^E_WARNING);
set_time_limit(0);
$parser = new ExceptionParser();
fwrite(STDOUT, "请输入日期，如 2017-05-10，默认为当天\n");
while(1){
    $dateRaw = trim(fgets(STDIN));
    $date = $dateRaw ? $dateRaw : date($logDateFormat);
    if(!preg_match('/^\d\d\d\d-\d\d-\d\d$/', $date)){
        fwrite(STDOUT, "日期格式错误，请重新输入\n");
        continue;
    }
    $path = $logPath.'/'.$logPrefix.$date.'.log';
    if(!file_exists($path)){
        $path = $logPath.'/'.$logPrefix.$date.'.log_';
        if(!file_exists($path)){
            fwrite(STDOUT, "日志文件不存在，请重新输入\n");
            continue;
        }
    }
    break;
}
fwrite(STDOUT, "请输入时间范围，格式如 01:00-12:00 如果为空则进行全局分析\n");
while(1){
    $timeRangeRaw = trim(fgets(STDIN));
    if($timeRangeRaw){
        $timeRange = explode('-', $timeRangeRaw);
        if(count($timeRange) != 2){
            fwrite(STDOUT, "时间范围格式有误，请重新输入\n");
            continue;
        }
        if(!preg_match('/^\d\d:\d\d$/', $timeRange[0]) || !preg_match('/^\d\d:\d\d$/', $timeRange[1])){
            fwrite(STDOUT, "时间范围格式有误，请重新输入\n");
            continue;
        }
        $parser->setTimeRange($timeRange[0], $timeRange[1]);
        break;
    }else{
        break;
    }
}
echo "分析中，请稍后 ...\n";
ob_start();
$parser->printNormalLogExceptionInfo($path);
$res = ob_get_clean();
$resFileName = $date.($timeRangeRaw ? '-'.$timeRangeRaw : '').'-exception.txt';
file_put_contents($resFileName, $res);
exit("分析结果已经保存在".$resFileName."文件中\n");

class ExceptionParser {
    
    const MINI_FORMAT = 'Y-m-d H:i';
    
    protected $data = [];
    
    private $beginTime;
    private $endTime;


    public function setTimeRange($begin = '', $end = ''){
        $this->beginTime = $begin;
        $this->endTime = $end;
    }
    
    //分析log中的异常情况
	public function printNormalLogExceptionInfo($filePath){
        $count_all = $this->vsistedCount($filePath);
        echo "有效访问总数：".$count_all." \n";
        //访问量最高统计
        $data_mini = $this->vsistedCountTopPerMini($filePath);
        $minis = array_keys($data_mini);
		$data_uas = $this->maxVsistedPerUrl($filePath, $minis, ['android', 'iphone']);
        $data_ips = $this->maxVsistedPerIpPerMini($filePath, $minis);
        $data_contentIds = $this->sortByUserParam($filePath, 'contentId', $minis, '/news/getContentDetails');
        $data_dts = $this->sortByUserParam($filePath, 'dt', $minis);
        $data_keywords = $this->sortByUserParam($filePath, 'keywords', $minis, '/news/search');
        echo "每分钟访问量TOP10：\n";
        $count_ = 10;
		foreach($data_mini as $mini => $count){
			echo "第".$mini."分钟接口分布，共".$count."个访问\n";
            echo "接口地址 TOP10：\n";
			$i = 0;
			foreach($data_uas[$mini] as $k => $v){
				if($i > $count_){
					break;
				}
				echo "总数：".$v["all"]."，android：".$v["android"]."，iphone：".$v["iphone"]."---".$k."\n";
				$i++;
			}
            echo "IP地址 TOP10：\n";
            $j = 0;
            foreach($data_ips[$mini] as $k => $v){
                if($j > $count_){
					break;
				}
				echo "IP：".$k."，总数：".$v."\n";
				$j++;
            }
            echo "新闻ID TOP10：\n";
            $l = 0;
            foreach($data_contentIds[$mini] as $k => $v){
                if($l > $count_){
					break;
				}
				echo "contentId：".$k."，总数：".$v."\n";
				$l++;
            }
            echo "设备ID TOP10：\n";
            $m = 0;
            foreach($data_dts[$mini] as $k => $v){
                if($m > $count_){
					break;
				}
				echo "dt：".$k."，总数：".$v."\n";
				$m++;
            }
            echo "搜索关键词 TOP10：\n";
            $k = 0;
            foreach($data_keywords[$mini] as $k => $v){
                if($k > $count_){
					break;
				}
				echo "关键词：".urldecode($k)."，总数：".$v."\n";
				$k++;
            }
            echo "\n";
		}
        echo "全局统计：\n";
        $data_all = [
            'IP' => $this->getVisitedCountByBaseParam($filePath, 'remote_add'),
            'UA' => $this->getVisitedCountByBaseParam($filePath, 'http_user_agent'),
            'Token' => $this->getVisitedCountByUserParam($filePath, 'token'),
            'DT' => $this->getVisitedCountByUserParam($filePath, 'dt'),
            'DK' => $this->getVisitedCountByUserParam($filePath, 'dk'),
        ];
        foreach($data_all as $name => $data){
            echo $name."访问量TOP10:\n";
            $i = 0;
            foreach($data as $param => $count){
                if($i >= 10){
                    break;
                }
                echo $param."：".$count."\n";
                $i ++;
            }
        }
	}
    
    //总数
	public function vsistedCount($filePath){
		$fd = fopen($filePath, 'r');
		$count = 0;
		while ($line = fgets($fd)){
			$logInfo = $this->parseLogFields($line);
			if(!isset($logInfo['request']['url'])){
				continue;
			}
			if($this->isStaticFile($logInfo['request']['url'])){
				continue;
			}
            if(!$this->isInTimeRange($logInfo['time_local'])){
                continue;
            }
			$count ++;
		}
        fclose($fd);
		return $count;
	}
	
	//每分钟最大请前十
	public function vsistedCountTopPerMini($filePath, $count = 10){
		$fd = fopen($filePath, 'r');
		$data_ = [];
		while ($line = fgets($fd)){
			$logInfo = $this->parseLogFields($line);
			if(!isset($logInfo['request']['url'])){
				continue;
			}
			if($this->isStaticFile($logInfo['request']['url'])){
				continue;
			}
            if(!$this->isInTimeRange($logInfo['time_local'])){
                continue;
            }
            $t_m = date(static::MINI_FORMAT, strtotime($logInfo['time_local']));
			if(isset($data_[$t_m])){
				$data_[$t_m] += 1;
			}else{
				$data_[$t_m] = 1;
			}
		}
        fclose($fd);
		$dataTemp = array_flip($data_);
		krsort($dataTemp);
        $i = 0;
		$minis = [];
		foreach(array_flip($dataTemp) as $k => $v){
			if($i > $count){
				break;
			}
			$minis[$k] = $v;
			$i++;
		}
		return $minis;
	}
    
    public function getTimeOutUrl($filePath, $seconds){
        $fd = fopen($filePath, 'r');
		$data_ = [];
		$timeOuts = [];
		while ($line = fgets($fd)){
			$logInfo = $this->parseLogFields($line);
			//请求时间超过指定时间的请求
			 $time = floatval(substr($line, -15, 5));
			 if($time > 5 && $logInfo['request']['url'] == '/news/search'){
				$t_m = date('Y-m-d H:i', strtotime($logInfo['time_local']));
				$paras = [];
				foreach($logInfo['request']['para'] as $key => $val){
					$paras[] = $key.':'.$val;
				}
				$timeOuts[] = 'time:'.$t_m.' '.$logInfo['request']['url'].' tt:'.floatval($time).' para:'.  implode(', ', $paras).' ip:'.$logInfo['remote_add'];
			}
			if(!isset($logInfo['request']['url'])){
				continue;
			}
			if($this->isStaticFile($logInfo['request']['url'])){
				continue;
			}
			$t_m = date('H:i', strtotime($logInfo['time_local']));
			if(isset($data_[$t_m])){
				$data_[$t_m] += 1;
			}else{
				$data_[$t_m] = 1;
			}
		}
		$data = array_flip($data_);
		krsort($data);
		fclose($fd);
		return $data;
    }
	
	//每分钟最大
	public function maxVsistedPerUrl($filePath, $minis, $uas = []){
        $minisArr = $this->getMinis($minis);
		$fd = fopen($filePath, 'r');
		$data_ = [];
		$data_uas = [];
		foreach($minisArr as $mini){
			$data_[$mini] = [];
			$data_uas[$mini] = [];
			foreach($uas as $ua){
				$data_uas[$mini][$ua] = [];
			}
		}
		while ($line = fgets($fd)){
			$logInfo = $this->parseLogFields($line);
			if(!isset($logInfo['request']['url'])){
				continue;
			}
			$t_m = date(static::MINI_FORMAT, strtotime($logInfo['time_local']));
			if(in_array($t_m, $minisArr)){
				$url = $logInfo['request']['url'];
				if($this->isStaticFile($url)){
					continue;
				}
				if(count($uas) > 0){
					$userAgent = $logInfo['http_user_agent'];
					foreach($uas as $ua){
						if(preg_match('/.*'.$ua.'.*/i', $userAgent)){
							if(isset($data_uas[$t_m][$ua][$url])){
								$data_uas[$t_m][$ua][$url] += 1;
							}else{
								$data_uas[$t_m][$ua][$url] = 1;
							}
						}
					}
				}
				if(isset($data_[$t_m][$url])){
					$data_[$t_m][$url] += 1;
				}else{
					$data_[$t_m][$url] = 1;
				}
			}
		}
		$data = [];
		foreach($data_ as $mini => $data_mini){
			$data__ = array_flip($data_mini);
			krsort($data__);
			foreach($data__ as $count => $url){
				$data[$mini][$url] = [
					'all' => $count
				];
				foreach($uas as $ua){
					$data[$mini][$url][$ua] = isset($data_uas[$mini][$ua][$url]) ? $data_uas[$mini][$ua][$url] : 0;
				}
			}
		}
		fclose($fd);
		return $data;
	}
    
    //每分钟请求时间最大
    public function maxVsistedPerIpPerMini($filePath, $minis = []){
        return $this->sortByBaseParam($filePath, 'remote_add', $minis);
	}
    
    //每分钟请求时间最大
	public function maxResponsTimePerMini($filePath, $minis){
        return $this->sortByBaseParam($filePath, 'respons_time', $minis);
	}
    
    //按照基础参数排序
    public function sortByBaseParam($filePath, $param, $minis){
        $fd = fopen($filePath, 'r');
		$data = [];
        $minisArr = $this->getMinis($minis);
		foreach($minisArr as $mini){
			$data[$mini] = [];
		}
		while ($line = fgets($fd)){
			$logInfo = $this->parseLogFields($line);
            if(!isset($logInfo['request']['url'])){
				continue;
			}
			if($this->isStaticFile($logInfo['request']['url'])){
				continue;
			}
			$t_m = date(static::MINI_FORMAT, strtotime($logInfo['time_local']));
			if(in_array($t_m, $minisArr)){
                $param_value = $logInfo[$param];
                if(!isset($data[$t_m][$param_value])){
                    $data[$t_m][$param_value] = 0;
                }
                $data[$t_m][$param_value] = $data[$t_m][$param_value] + 1;
			}
		}
		foreach($data as $mini => $data_mini){
			$data__ = array_flip($data_mini);
			krsort($data__);
            $data[$mini] = array_flip($data__);
		}
		fclose($fd);
		return $data;
    }
    
    //按照基础参数排序
    public function sortByUserParam($filePath, $param, $minis, $url = ''){
        $fd = fopen($filePath, 'r');
		$data = [];
        $minisArr = $this->getMinis($minis);
		foreach($minisArr as $mini){
			$data[$mini] = [];
		}
		while ($line = fgets($fd)){
			$logInfo = $this->parseLogFields($line);
            if(!isset($logInfo['request']['url'])){
				continue;
			}
			if($this->isStaticFile($logInfo['request']['url'])){
				continue;
			}
			$t_m = date(static::MINI_FORMAT, strtotime($logInfo['time_local']));
            if(!in_array($t_m, $minisArr)){
                continue;
            }
            if($url && trim($logInfo['request']['url']) !== $url){
                continue;
            }
            if(!isset($logInfo['request']['para'][$param])){
                continue;
            }
			$param_value = $logInfo['request']['para'][$param];
            if(!isset($data[$t_m][$param_value])){
                $data[$t_m][$param_value] = 0;
            }
            $data[$t_m][$param_value] = $data[$t_m][$param_value] + 1;
		}
		foreach($data as $mini => $data_mini){
			$data__ = array_flip($data_mini);
			krsort($data__);
            $data[$mini] = array_flip($data__);
		}
		fclose($fd);
		return $data;
    }
    
    //获取指定参数的访问量
    public function getVisitedCountByBaseParam($filePath, $param){
        $fd = fopen($filePath, 'r');
		$data = [];
		while ($line = fgets($fd)){
			$logInfo = $this->parseLogFields($line);
            if(!isset($logInfo['request']['url'])){
				continue;
			}
			if($this->isStaticFile($logInfo['request']['url'])){
				continue;
			}
            if(!isset($logInfo[$param])){
                continue;
            }
            if(!$this->isInTimeRange($logInfo['time_local'])){
                continue;
            }
            $param_value = $logInfo[$param];
            if(!isset($data[$param_value])){
                $data[$param_value] = 0;
            }
            $data[$param_value] = $data[$param_value] + 1;
		}
        fclose($fd);
        $data_ = array_flip($data);
        krsort($data_);
		return array_flip($data_);
    }
    
    //获取指定参数的访问量
    public function getVisitedCountByUserParam($filePath, $param, $url = ''){
        $fd = fopen($filePath, 'r');
		$data = [];
		while ($line = fgets($fd)){
			$logInfo = $this->parseLogFields($line);
            if(!isset($logInfo['request']['url'])){
				continue;
			}
			if($this->isStaticFile($logInfo['request']['url'])){
				continue;
			}
            if($url && trim($logInfo['request']['url']) !== $url){
                continue;
            }
            if(!isset($logInfo['request']['para'][$param])){
                continue;
            }
            if(!$this->isInTimeRange($logInfo['time_local'])){
                continue;
            }
            $param_value = $logInfo['request']['para'][$param];
            if(!isset($data[$param_value])){
                $data[$param_value] = 0;
            }
            $data[$param_value] = $data[$param_value] + 1;
		}
        fclose($fd);
        $data_ = array_flip($data);
        krsort($data_);
		return array_flip($data_);
    }
    
    //获取分钟数组
    private function getMinis($minis){
        if(is_array($minis)){
            return $minis;
        }
        switch (strlen($minis)){
            case 10 :
                $minisArr = [];
                for($i = 0;$i<24;$i++){
                    for($j=0;$j<60;$j++){
                        $minisArr[] = $minis.' '.($i < 10 ? '0'.$i : $i).':'.($j < 10 ? '0'.$j : $j);
                    }
                }
                break;
            case 13 :
                $minisArr = [];
                for($j=0;$j<60;$j++){
                    $minisArr[] = $minis.':'.($j < 10 ? '0'.$j : $j);
                }
                break;
            default :
                return [];
        }
        return $minisArr;
    }
    
    //解析Nginx log字段
	private function parseLogFields($logItem){
		$matches = $matches1 = $matches2 = [];
		preg_match('/(.*?)-(.*?)-\s*\[(.*?)\].*?"(.*?)\"\s*(\d*)\s*(\d*)\s*"(.*?)\"\s*\"(.*?)\".*urt:"(.*?)".*rt:"(.*?)"/', $logItem, $matches);
		if(count($matches) === 0){
			return [];
		}
		//分析请求
		$request = ['method' => '', 'url' => '', 'para' => [], 'prot' => ''];
		preg_match('/(.*?)\s+(.*?)\s+(.*)/', trim($matches[4]), $matches1);
		if(count($matches1) === 0){
			return [];
		}
		$url = html_entity_decode(trim($matches1[2]));
		$para = [];
		preg_match('/(.*?)\?(.*)/', $url, $matches2);
		if(count($matches2) > 0){
			$url = $matches2[1];
			$paramArr = explode('&', $matches2[2]);
			foreach ($paramArr as $paramStr){
				if(strpos($paramStr, '=') === false){
					continue;
				}
				$paramArr_ = explode('=', $paramStr);
				$para[$paramArr_[0]] = $paramArr_[1];
			}
		}
		$request['method'] = $matches1[1];
		$request['url'] = $url;
		$request['para'] = $para;
		$request['prot'] = $matches1[3];
		return [
			'remote_add' => trim($matches[1]),
			'remote_user' => trim($matches[2]),
			'time_local' => trim($matches[3]),
			'request' => $request,
			'status' => trim($matches[5]),
			'body_bytes_sent' => trim($matches[6]),
			'http_referer' => trim($matches[7]),
			'http_user_agent' => trim($matches[8]),
            'response_time' => floatval($matches[9]),
            'request_time' => floatval($matches[10])
		];
	}
    
    //判断静态文件接口
	private function isStaticFile($url){
        return strpos($url, '.') !== false;
	}
    
    //判断是否在事件范围之内
    private function isInTimeRange($time){
        if(!$this->beginTime || !$this->endTime){
            return true;
        }
        $d_m = date('Y-m-d', strtotime($time));
        $t_m = date(static::MINI_FORMAT, strtotime($time));
        if($t_m < $d_m.' '.$this->beginTime || $t_m > $d_m.' '.$this->endTime){
            return false;
        }
        return true;
    }
	
	//分析指定新闻的信息
	public function testContentInfo($logFile){
		if(!file_exists($logFile)){
			return false;
		}
		set_time_limit(0);
		$fd = fopen($logFile, 'r');
		$data = ['click' => 0];
		while ($line = fgets($fd)){
			$logInfo = $this->parseLogFields($line);
			if(!isset($logInfo['request']['url'])){
				continue;
			}
			if(strpos($logInfo['request']['url'], 'news/getContentDetails') === false){
				continue;
			}
			if($this->isTheUrl($logInfo, 'news/getContentDetails', ['type', 'contentId'])){//新闻点击量
				$from = 'c_0';//未知来源
				if(isset($logInfo['request']['para']['c'])){//来自分享
					$type = $logInfo['request']['para']['c'];
					if(in_array($type, ['1', '2', '3', '4', '5', '6', '7'], true)){
						$from = 'c_'.$type;
					}
				}else{//来自客户端
					$userAgent = $logInfo['http_user_agent'];
					if(preg_match('/.*android.*/i', $userAgent)){
						$from = 'ty_a';
					}elseif(preg_match('/.*iphone.*/i', $userAgent)){
						$from = 'ty_i';
					}
				}
				if($logInfo['request']['para']['contentId'] == '5771435b87ea8320932176'){
					$data['click'] = $data['click'] + 1;
					if(!isset($data[$from])){
						$data[$from] = 0;
					}
					$data[$from] = $data[$from] + 1;
				}
			}
		}
		fclose($fd);
		echo '<pre>';
		print_r($data);
	}
}